<!DOCTYPE html>
<html lang="zh-CN">
    <head>
        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1">
        <meta name="robots" content="noodp" />
        <meta http-equiv="X-UA-Compatible" content="IE=edge, chrome=1">
        <title>04_K8S_核心网络插件Flannel - 山脚下的脚下山</title><meta name="Description" content="转载，原为老男孩教育视频内容"><meta property="og:title" content="04_K8S_核心网络插件Flannel" />
<meta property="og:description" content="转载，原为老男孩教育视频内容" />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://scemsjyd.com/04_k8s_flannel/" /><meta property="og:image" content="https://scemsjyd.com/logo.png"/><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2020-10-01T15:58:21+08:00" />
<meta property="article:modified_time" content="2020-10-01T15:58:21+08:00" />

<meta name="twitter:card" content="summary_large_image"/>
<meta name="twitter:image" content="https://scemsjyd.com/logo.png"/>

<meta name="twitter:title" content="04_K8S_核心网络插件Flannel"/>
<meta name="twitter:description" content="转载，原为老男孩教育视频内容"/>
<meta name="application-name" content="山脚下の脚下山">
<meta name="apple-mobile-web-app-title" content="山脚下の脚下山"><meta name="theme-color" content="#ffffff"><meta name="msapplication-TileColor" content="#da532c"><link rel="shortcut icon" type="image/x-icon" href="/favicon.ico" />
        <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
        <link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"><link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png"><link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5"><link rel="manifest" href="/site.webmanifest"><link rel="canonical" href="https://scemsjyd.com/04_k8s_flannel/" /><link rel="prev" href="https://scemsjyd.com/05_k8s_%E6%A0%B8%E5%BF%83%E6%8F%92%E4%BB%B6coredns%E6%9C%8D%E5%8A%A1/" /><link rel="next" href="https://scemsjyd.com/03_k8s_%E7%AE%A1%E7%90%86k8s%E6%A0%B8%E5%BF%83%E8%B5%84%E6%BA%90%E7%9A%84%E4%B8%89%E7%A7%8D%E5%9F%BA%E6%9C%AC%E6%96%B9%E6%B3%95/" /><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/normalize.css@8.0.1/normalize.min.css"><link rel="stylesheet" href="/css/style.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@5.13.0/css/all.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/animate.css@3.7.2/animate.min.css"><script type="application/ld+json">
    {
        "@context": "http://schema.org",
        "@type": "BlogPosting",
        "headline": "04_K8S_核心网络插件Flannel",
        "inLanguage": "zh-CN",
        "mainEntityOfPage": {
            "@type": "WebPage",
            "@id": "https:\/\/scemsjyd.com\/04_k8s_flannel\/"
        },"image": ["https:\/\/scemsjyd.com\/images\/Apple-Devices-Preview.png"],"genre": "posts","keywords": "K8S, 转载","wordcount":  2876 ,
        "url": "https:\/\/scemsjyd.com\/04_k8s_flannel\/","datePublished": "2020-10-01T15:58:21+08:00","dateModified": "2020-10-01T15:58:21+08:00","license": "This work is licensed under a Creative Commons Attribution-NonCommercial 4.0 International License.","publisher": {
            "@type": "Organization",
            "name": "Adam.Jin","logo": "https:\/\/p.qlogo.cn\/bizmail\/XpHAlE90tNCBxpvcMjtmdG52qxW6TFrUQ3VbWchBMibqlYJup0yhBzQ\/"},"author": {
                "@type": "Person",
                "name": "老男孩"
            },"description": "转载，原为老男孩教育视频内容"
    }
    </script></head>
    <body header-desktop="fixed" header-mobile="auto"><script type="text/javascript">(window.localStorage && localStorage.getItem('theme') ? localStorage.getItem('theme') === 'dark' : ('auto' === 'auto' ? window.matchMedia('(prefers-color-scheme: dark)').matches : 'auto' === 'dark')) && document.body.setAttribute('theme', 'dark');</script>

        <div id="mask"></div><div class="wrapper"><header class="desktop" id="header-desktop">
    <div class="header-wrapper">
        <div class="header-title">
            <a href="/" title="山脚下的脚下山"><span class="header-title-pre"><i class='far fa-kiss-wink-heart fa-fw'></i></span>山脚下の脚下山</a>
        </div>
        <div class="menu">
            <div class="menu-inner"><a class="menu-item" href="/"> 主页 </a><a class="menu-item" href="/posts/"> 文章 </a><a class="menu-item" href="/tags/"> 标签 </a><a class="menu-item" href="/categories/"> 分类 </a><a class="menu-item" href="/about/"> 关于 </a><a class="menu-item" href="https://github.com/scemsjyd" title="GitHub" rel="noopener noreffer" target="_blank"><i class='fab fa-github fa-fw'></i>  </a><span class="menu-item delimiter"></span><span class="menu-item search" id="search-desktop">
                        <input type="text" placeholder="搜索文章标题或内容..." id="search-input-desktop">
                        <a href="javascript:void(0);" class="search-button search-toggle" id="search-toggle-desktop" title="搜索">
                            <i class="fas fa-search fa-fw"></i>
                        </a>
                        <a href="javascript:void(0);" class="search-button search-clear" id="search-clear-desktop" title="清空">
                            <i class="fas fa-times-circle fa-fw"></i>
                        </a>
                        <span class="search-button search-loading" id="search-loading-desktop">
                            <i class="fas fa-spinner fa-fw fa-spin"></i>
                        </span>
                    </span><a href="javascript:void(0);" class="menu-item theme-switch" title="切换主题">
                    <i class="fas fa-adjust fa-fw"></i>
                </a>
            </div>
        </div>
    </div>
</header><header class="mobile" id="header-mobile">
    <div class="header-container">
        <div class="header-wrapper">
            <div class="header-title">
                <a href="/" title="山脚下的脚下山"><span class="header-title-pre"><i class='far fa-kiss-wink-heart fa-fw'></i></span>山脚下の脚下山</a>
            </div>
            <div class="menu-toggle" id="menu-toggle-mobile">
                <span></span><span></span><span></span>
            </div>
        </div>
        <div class="menu" id="menu-mobile"><div class="search-wrapper">
                    <div class="search mobile" id="search-mobile">
                        <input type="text" placeholder="搜索文章标题或内容..." id="search-input-mobile">
                        <a href="javascript:void(0);" class="search-button search-toggle" id="search-toggle-mobile" title="搜索">
                            <i class="fas fa-search fa-fw"></i>
                        </a>
                        <a href="javascript:void(0);" class="search-button search-clear" id="search-clear-mobile" title="清空">
                            <i class="fas fa-times-circle fa-fw"></i>
                        </a>
                        <span class="search-button search-loading" id="search-loading-mobile">
                            <i class="fas fa-spinner fa-fw fa-spin"></i>
                        </span>
                    </div>
                    <a href="javascript:void(0);" class="search-cancel" id="search-cancel-mobile">
                        取消
                    </a>
                </div><a class="menu-item" href="/" title="">主页</a><a class="menu-item" href="/posts/" title="">文章</a><a class="menu-item" href="/tags/" title="">标签</a><a class="menu-item" href="/categories/" title="">分类</a><a class="menu-item" href="/about/" title="">关于</a><a class="menu-item" href="https://github.com/scemsjyd" title="GitHub" rel="noopener noreffer" target="_blank"><i class='fab fa-github fa-fw'></i></a><a href="javascript:void(0);" class="menu-item theme-switch" title="切换主题">
                <i class="fas fa-adjust fa-fw"></i>
            </a></div>
    </div>
</header>
<div class="search-dropdown desktop">
    <div id="search-dropdown-desktop"></div>
</div>
<div class="search-dropdown mobile">
    <div id="search-dropdown-mobile"></div>
</div>
<main class="main">
                <div class="container"><div class="toc" id="toc-auto">
            <h2 class="toc-title">目录</h2>
            <div class="toc-content" id="toc-content-auto"></div>
        </div><article class="page single"><h1 class="single-title animated flipInX">04_K8S_核心网络插件Flannel</h1><div class="post-meta">
            <div class="post-meta-line"><span class="post-author"><a href="/" title="Author" rel=" author" class="author"><i class="fas fa-user-circle fa-fw"></i>老男孩</a></span>&nbsp;<span class="post-category">收录于 <a href="/categories/%E8%BD%AC%E8%BD%BD/"><i class="far fa-folder fa-fw"></i>转载</a>&nbsp;<a href="/categories/k8s/"><i class="far fa-folder fa-fw"></i>K8S</a></span></div>
            <div class="post-meta-line"><i class="far fa-calendar-alt fa-fw"></i>&nbsp;<time datetime="2020-10-01">2020-10-01</time>&nbsp;<i class="fas fa-pencil-alt fa-fw"></i>&nbsp;约 2876 字&nbsp;
                <i class="far fa-clock fa-fw"></i>&nbsp;预计阅读 6 分钟&nbsp;<span id="/04_k8s_flannel/" class="leancloud_visitors" data-flag-title="04_K8S_核心网络插件Flannel">
                        <i class="far fa-eye fa-fw"></i>&nbsp;<span class=leancloud-visitors-count></span>&nbsp;次阅读
                    </span>&nbsp;</div>
        </div><div class="featured-image"><img
        class="lazyload"
        src="/svg/loading.min.svg"
        data-src="https://upyun.cdn.scemsjyd.com/PicGo/2022/03/20220310184017.png"
        data-srcset="https://upyun.cdn.scemsjyd.com/PicGo/2022/03/20220310184017.png, https://upyun.cdn.scemsjyd.com/PicGo/2022/03/20220310184017.png 1.5x, https://upyun.cdn.scemsjyd.com/PicGo/2022/03/20220310184017.png 2x"
        data-sizes="auto"
        alt="https://upyun.cdn.scemsjyd.com/PicGo/2022/03/20220310184017.png"
        title="转载，原为老男孩教育视频内容" /></div><div class="details toc" id="toc-static"  kept="">
                <div class="details-summary toc-title">
                    <span>目录</span>
                    <span><i class="details-icon fas fa-angle-right"></i></span>
                </div>
                <div class="details-content toc-content" id="toc-content-static"><nav id="TableOfContents">
  <ul>
    <li><a href="#1--flannel功能概述">1.  flannel功能概述</a>
      <ul>
        <li><a href="#11-flannel运转流程">1.1 flannel运转流程</a></li>
        <li><a href="#12-flannel的网络模型">1.2 flannel的网络模型</a>
          <ul>
            <li><a href="#121-flannel支持3种网络模型">1.2.1 flannel支持3种网络模型</a></li>
            <li><a href="#122-实际工作中的模型选择">1.2.2 实际工作中的模型选择</a></li>
          </ul>
        </li>
      </ul>
    </li>
    <li><a href="#2-部署flannel插件">2. 部署flannel插件</a>
      <ul>
        <li><a href="#21-在etcd中写入网络信息">2.1 在etcd中写入网络信息</a></li>
        <li><a href="#22-部署准备">2.2 部署准备</a>
          <ul>
            <li><a href="#221-下载软件">2.2.1 下载软件</a></li>
            <li><a href="#222-拷贝证书">2.2.2 拷贝证书</a></li>
            <li><a href="#223-配置子网信息">2.2.3 配置子网信息</a></li>
          </ul>
        </li>
        <li><a href="#23-启动flannel服务">2.3 启动flannel服务</a>
          <ul>
            <li><a href="#231-创建flannel启动脚本">2.3.1 创建flannel启动脚本</a></li>
            <li><a href="#232-创建supervisor启动脚本">2.3.2 创建supervisor启动脚本</a></li>
            <li><a href="#233-启动flannel服务并验证">2.3.3 启动flannel服务并验证</a></li>
          </ul>
        </li>
      </ul>
    </li>
    <li><a href="#3-优化iptables规则">3 优化iptables规则</a>
      <ul>
        <li><a href="#31-前因后果">3.1 前因后果</a>
          <ul>
            <li><a href="#311-优化原因说明">3.1.1 优化原因说明</a></li>
            <li><a href="#312-问题复现">3.1.2 问题复现</a></li>
          </ul>
        </li>
        <li><a href="#32-具体优化过程">3.2 具体优化过程</a>
          <ul>
            <li><a href="#321-先查看iptables规则">3.2.1 先查看iptables规则</a></li>
            <li><a href="#322-安装iptables并修改规则">3.2.2 安装iptables并修改规则</a></li>
            <li><a href="#323-注意docker重启后操作">3.2.3 注意docker重启后操作</a></li>
            <li><a href="#324-结果验证">3.2.4 结果验证</a></li>
          </ul>
        </li>
      </ul>
    </li>
  </ul>
</nav></div>
            </div><div class="content" id="content"><h1 id="04_k8s_核心网络插件flannel">04_K8S_核心网络插件Flannel</h1>
<blockquote>
<p>k8s虽然设计了网络模型,然后将实现方式交给了CNI网络插件,而CNI网络插件的主要目的,就是实现POD资源能够跨宿主机进行通信
常见的网络插件有<code>flannel</code>,<code>calico</code>,<code>canal</code>,但是最简单的<code>flannel</code>已经完全满足我们的要求,故不在考虑其他网络插件
<strong>网络插件Flannel介绍：<a href="https://www.kubernetes.org.cn/3682.html" target="_blank" rel="noopener noreffer">https://www.kubernetes.org.cn/3682.html</a></strong></p>
</blockquote>
<h2 id="1--flannel功能概述">1.  flannel功能概述</h2>
<h3 id="11-flannel运转流程">1.1 flannel运转流程</h3>
<ol>
<li>首先
flannel利用Kubernetes API或者etcd用于存储整个集群的网络配置，其中最主要的内容为设置集群的网络地址空间。
例如，设定整个集群内所有容器的IP都取自网段“10.1.0.0/16”。</li>
<li>接着
flannel在每个主机中运行flanneld作为agent，它会为所在主机从集群的网络地址空间中，获取一个小的网段subnet，本主机内所有容器的IP地址都将从中分配。
例如，设定本主机内所有容器的IP地址网段“10.1.2.0/24”。</li>
<li>然后
flanneld再将本主机获取的subnet以及用于主机间通信的Public IP，同样通过kubernetes API或者etcd存储起来。</li>
<li>最后
flannel利用各种backend mechanism，例如udp，vxlan等等，跨主机转发容器间的网络流量，完成容器间的跨主机通信。</li>
</ol>
<h3 id="12-flannel的网络模型">1.2 flannel的网络模型</h3>
<h4 id="121-flannel支持3种网络模型">1.2.1 flannel支持3种网络模型</h4>
<ol>
<li><code>host-gw</code>网关模型</li>
</ol>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">{&#34;Network&#34;: &#34;xxx&#34;, &#34;Backend&#34;: {&#34;Type&#34;: &#34;host-gw&#34;}}
</code></pre></td></tr></table>
</div>
</div><ol>
<li>主要用于宿主机在同网段的情况下POD间的通信,即不跨网段通信.
此时flannel的功能很简单,就是在每个宿主机上创建了一条通网其他宿主机的网关路由
完全没有性能损耗,效率极高</li>
<li>vxlan隧道模型</li>
</ol>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">{&#34;Network&#34;: &#34;xxx&#34;, &#34;Backend&#34;: {&#34;Type&#34;: &#34;vxlan&#34;}}
</code></pre></td></tr></table>
</div>
</div><ol>
<li>主要用于宿主机不在同网段的情况下POD间通信,即跨网段通信.
此时flannel会在宿主机上创建一个<code>flannel.1</code>的虚拟网卡,用于和其他宿主机间建立VXLAN隧道
跨宿主机通信时,需要经由<code>flannel.1</code>设备封包、解包，因此效率不高</li>
<li>混合模型</li>
</ol>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">{&#34;Network&#34;: &#34;xxx&#34;, &#34;Backend&#34;: {&#34;Type&#34;: &#34;vxlan&#34;,&#34;Directrouting&#34;: true}}
</code></pre></td></tr></table>
</div>
</div><ol>
<li>在既有同网段宿主机，又有跨网段宿主机的情况下，选择混合模式
flannel会根据通信双方的网段情况，自动选择是走网关路由通信还是通过VXLAN隧道通信</li>
</ol>
<h4 id="122-实际工作中的模型选择">1.2.2 实际工作中的模型选择</h4>
<p>很多人不推荐部署K8S的使用的flannel做网络插件,不推荐的原因是是flannel性能不高,然而</p>
<ol>
<li>flannel性能不高是指它的VXLAN隧道模型,而不是gw模型</li>
<li>规划K8S集群的时候,应规划多个K8S集群来管理不同的业务</li>
<li>同一个K8S集群的宿主机,就应该规划到同一个网段</li>
<li>既然是同一个网段的宿主机通信,使用的就应该是gw模型</li>
<li>gw模型只是创建了网关路由,通信效率极高</li>
<li>因此,建议工作中使用flannel,且用<code>gw</code>模型</li>
</ol>
<h2 id="2-部署flannel插件">2. 部署flannel插件</h2>
<h3 id="21-在etcd中写入网络信息">2.1 在etcd中写入网络信息</h3>
<p>以下操作在任意etcd节点中执行都可以</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">/opt/etcd/etcdctl set /coreos.com/network/config &#39;{&#34;Network&#34;: &#34;172.7.0.0/16&#34;, &#34;Backend&#34;: {&#34;Type&#34;: &#34;host-gw&#34;}}&#39;
# 查看结果
[root@hdss7-12 ~]# /opt/etcd/etcdctl get /coreos.com/network/config
{&#34;Network&#34;: &#34;172.7.0.0/16&#34;, &#34;Backend&#34;: {&#34;Type&#34;: &#34;host-gw&#34;}}
</code></pre></td></tr></table>
</div>
</div><h3 id="22-部署准备">2.2 部署准备</h3>
<h4 id="221-下载软件">2.2.1 下载软件</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
mkdir /opt/flannel-v0.11.0
tar xf flannel-v0.11.0-linux-amd64.tar.gz -C /opt/flannel-v0.11.0/
ln -s /opt/flannel-v0.11.0/ /opt/flannel
</code></pre></td></tr></table>
</div>
</div><h4 id="222-拷贝证书">2.2.2 拷贝证书</h4>
<p>因为要和apiserver通信，所以要配置client证书,当然ca公钥自不必说</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cd /opt/flannel
mkdir cert
scp hdss7-200:/opt/certs/ca.pem         cert/
scp hdss7-200:/opt/certs/client.pem     cert/
scp hdss7-200:/opt/certs/client-key.pem cert/
</code></pre></td></tr></table>
</div>
</div><h4 id="223-配置子网信息">2.2.3 配置子网信息</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;/opt/flannel/subnet.env &lt;&lt;EOF
FLANNEL_NETWORK=172.7.0.0/16
FLANNEL_SUBNET=172.7.21.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false
EOF
</code></pre></td></tr></table>
</div>
</div><blockquote>
<p>注意:subnet子网网段信息,每个宿主机都要修改</p>
</blockquote>
<h3 id="23-启动flannel服务">2.3 启动flannel服务</h3>
<h4 id="231-创建flannel启动脚本">2.3.1 创建flannel启动脚本</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash">cat &gt;/opt/flannel/flanneld.sh <span class="s">&lt;&lt;&#39;EOF&#39;
</span><span class="s">#!/bin/sh
</span><span class="s">./flanneld \
</span><span class="s">  --public-ip=10.4.7.21 \
</span><span class="s">  --etcd-endpoints=https://10.4.7.12:2379,https://10.4.7.21:2379,https://10.4.7.22:2379 \
</span><span class="s">  --etcd-keyfile=./cert/client-key.pem \
</span><span class="s">  --etcd-certfile=./cert/client.pem \
</span><span class="s">  --etcd-cafile=./cert/ca.pem \
</span><span class="s">  --iface=eth0 \
</span><span class="s">  --subnet-file=./subnet.env \
</span><span class="s">  --healthz-port=2401
</span><span class="s">EOF</span>
<span class="c1"># 授权</span>
chmod u+x flanneld.sh
</code></pre></td></tr></table>
</div>
</div><blockquote>
<p>注意:
public-ip为节点IP,注意按需修改
iface为网卡,若本机网卡不是eth0,注意修改</p>
</blockquote>
<h4 id="232-创建supervisor启动脚本">2.3.2 创建supervisor启动脚本</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">cat &gt;/etc/supervisord.d/flannel.ini &lt;&lt;EOF
[program:flanneld]
command=sh /opt/flannel/flanneld.sh
numprocs=1
directory=/opt/flannel
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/flanneld/flanneld.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=4
stdout_capture_maxbytes=1MB
;子进程还有子进程,需要添加这个参数,避免产生孤儿进程
killasgroup=true
stopasgroup=true
EOF
</code></pre></td></tr></table>
</div>
</div><blockquote>
<p>supervisor的各项配置不再备注,有需要的看K8S二进制安装中的备注</p>
</blockquote>
<h4 id="233-启动flannel服务并验证">2.3.3 启动flannel服务并验证</h4>
<p><strong>启动服务</strong></p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">mkdir -p /data/logs/flanneld
supervisorctl update
supervisorctl status
</code></pre></td></tr></table>
</div>
</div><p><strong>验证路由</strong></p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span><span class="lnt">7
</span><span class="lnt">8
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">[root@hdss7-22 ~]# route -n|egrep -i &#39;172.7|des&#39;
Destination   Gateway     Genmask         Flags Metric Ref   Use Iface
172.7.21.0    10.4.7.21   255.255.255.0   UG    0      0       0 eth0
172.7.22.0    0.0.0.0     255.255.255.0   U     0      0       0 docker0
[root@hdss7-21 ~]# route -n|egrep -i &#39;172.7|des&#39;
Destination   Gateway     Genmask         Flags Metric Ref   Use Iface
172.7.21.0    0.0.0.0     255.255.255.0   U     0      0       0 docker0
172.7.22.0    10.4.7.22   255.255.255.0   UG    0      0       0 eth0
</code></pre></td></tr></table>
</div>
</div><p><strong>验证通信结果</strong></p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span><span class="lnt">7
</span><span class="lnt">8
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">[root@hdss7-21 ~]# ping 172.7.22.2
PING 172.7.22.2 (172.7.22.2) 56(84) bytes of data.
64 bytes from 172.7.22.2: icmp_seq=1 ttl=63 time=0.538 ms
64 bytes from 172.7.22.2: icmp_seq=2 ttl=63 time=0.896 ms
[root@hdss7-22 ~]# ping 172.7.21.2
PING 172.7.21.2 (172.7.21.2) 56(84) bytes of data.
64 bytes from 172.7.21.2: icmp_seq=1 ttl=63 time=0.805 ms
64 bytes from 172.7.21.2: icmp_seq=2 ttl=63 time=1.14 ms
</code></pre></td></tr></table>
</div>
</div><h2 id="3-优化iptables规则">3 优化iptables规则</h2>
<h3 id="31-前因后果">3.1 前因后果</h3>
<h4 id="311-优化原因说明">3.1.1 优化原因说明</h4>
<p>我们使用的是<code>gw</code>网络模型,而这个网络模型只是创建了一条到其他宿主机下POD网络的路由信息.
<strong>因而我们可以猜想:</strong></p>
<ol>
<li>从外网访问到B宿主机中的POD,源IP应该是外网IP</li>
<li>从A宿主机访问B宿主机中的POD,源IP应该是A宿主机的IP</li>
<li>从A的POD-A01中,访问B中的POD,源IP应该是POD-A01的容器IP
此情形可以想象是一个路由器下的2个不同网段的交换机下的设备通过路由器(gw)通信</li>
</ol>
<p><strong>然后遗憾的是:</strong></p>
<ul>
<li>前两条毫无疑问成立</li>
<li>第3条理应成立,但实际不成立</li>
</ul>
<p><strong>不成立的原因是:</strong></p>
<ol>
<li>Docker容器的跨网络隔离与通信，借助了iptables的机制</li>
<li>因此虽然K8S我们使用了ipvs调度,但是宿主机上还是有iptalbes规则</li>
<li>而docker默认生成的iptables规则为：
若数据出网前，先判断出网设备是不是本机<code>docker0</code>设备(容器网络)
如果不是的话，则进行SNAT转换后再出网，具体规则如下</li>
</ol>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">[root@hdss7-21 ~]# iptables-save |grep -i postrouting|grep docker0
-A POSTROUTING -s 172.7.21.0/24 ! -o docker0 -j MASQUERADE
</code></pre></td></tr></table>
</div>
</div><ol>
<li>由于<code>gw</code>模式产生的数据,是从<code>eth0</code>流出,因而不在此规则过滤范围内</li>
<li>就导致此跨宿主机之间的POD通信,使用了该条SNAT规则</li>
</ol>
<p><strong>解决办法是:</strong></p>
<ul>
<li>修改此IPTABLES规则,增加过滤目标:<strong>过滤目的地是宿主机网段的流量</strong></li>
</ul>
<h4 id="312-问题复现">3.1.2 问题复现</h4>
<ol>
<li>在<code>7-21</code>宿主机中,访问<code>172.7.22.2</code></li>
</ol>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">curl 172.7.22.2
</code></pre></td></tr></table>
</div>
</div><ol>
<li>在<code>7-21</code>宿主机启动busybox容器,进入并访问<code>172.7.22.2</code></li>
</ol>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">docker pull busybox
docker run --rm -it busybox bash
/ # wget 172.7.22.2
</code></pre></td></tr></table>
</div>
</div><ol>
<li>查看<code>7-22</code>宿主机上启动的nginx容器日志</li>
</ol>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">[root@hdss7-22 ~]# kubectl logs nginx-ds-j777c --tail=2
10.4.7.21 - - [xxx] &#34;GET / HTTP/1.1&#34; 200 612 &#34;-&#34; &#34;curl/7.29.0&#34; &#34;-&#34;
10.4.7.21 - - [xxx] &#34;GET / HTTP/1.1&#34; 200 612 &#34;-&#34; &#34;Wget&#34; &#34;-&#34;
</code></pre></td></tr></table>
</div>
</div><ol>
<li>第一条日志为对端宿主机访问日志
第二条日志为对端容器访问日志
可以看出源IP都是宿主机的IP</li>
</ol>
<h3 id="32-具体优化过程">3.2 具体优化过程</h3>
<h4 id="321-先查看iptables规则">3.2.1 先查看iptables规则</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">[root@hdss7-21 ~]# iptables-save |grep -i postrouting|grep docker0
-A POSTROUTING -s 172.7.21.0/24 ! -o docker0 -j MASQUERADE
</code></pre></td></tr></table>
</div>
</div><h4 id="322-安装iptables并修改规则">3.2.2 安装iptables并修改规则</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span><span class="lnt">7
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">yum install iptables-services -y
iptables -t nat -D POSTROUTING -s 172.7.21.0/24 ! -o docker0 -j MASQUERADE
iptables -t nat -I POSTROUTING -s 172.7.21.0/24 ! -d 172.7.0.0/16 ! -o docker0  -j MASQUERADE
# 验证规则并保存配置
[root@hdss7-21 ~]# iptables-save |grep -i postrouting|grep docker0
-A POSTROUTING -s 172.7.21.0/24 ! -d 172.7.0.0/16 ! -o docker0 -j MASQUERADE
[root@hdss7-21 ~]# iptables-save &gt; /etc/sysconfig/iptables
</code></pre></td></tr></table>
</div>
</div><h4 id="323-注意docker重启后操作">3.2.3 注意docker重启后操作</h4>
<p>docker服务重启后,会再次增加该规则,要注意在每次重启docker服务后,删除该规则
验证:
修改后会影响到docker原本的iptables链的规则，所以需要重启docker服务</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span><span class="lnt">7
</span><span class="lnt">8
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback">[root@hdss7-21 ~]# systemctl restart docker
[root@hdss7-21 ~]# iptables-save |grep -i postrouting|grep docker0
-A POSTROUTING -s 172.7.21.0/24 ! -o docker0 -j MASQUERADE
-A POSTROUTING -s 172.7.21.0/24 ! -d 172.7.0.0/16 ! -o docker0 -j MASQUERADE
# 可以用iptables-restore重新应用iptables规则,也可以直接再删
[root@hdss7-21 ~]# iptables-restore /etc/sysconfig/iptables
[root@hdss7-21 ~]# iptables-save |grep -i postrouting|grep docker0
-A POSTROUTING -s 172.7.21.0/24 ! -d 172.7.0.0/16 ! -o docker0 -j MASQUERADE
</code></pre></td></tr></table>
</div>
</div><h4 id="324-结果验证">3.2.4 结果验证</h4>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-fallback" data-lang="fallback"># 对端启动容器并访问
[root@hdss7-21 ~]# docker run --rm -it busybox  sh
/ # wget 172.7.22.2
# 本端验证日志
[root@hdss7-22 ~]# kubectl logs nginx-ds-j777c --tail=1
172.7.21.3 - - [xxxx] &#34;GET / HTTP/1.1&#34; 200 612 &#34;-&#34; &#34;Wget&#34; &#34;-&#34;
</code></pre></td></tr></table>
</div>
</div></div><div class="post-footer" id="post-footer">
    <div class="post-info">
        <div class="post-info-line">
            <div class="post-info-mod">
                <span>更新于 2020-10-01</span>
            </div>
            <div class="post-info-license"></div>
        </div>
        <div class="post-info-line">
            <div class="post-info-md"></div>
            <div class="post-info-share">
                <span><a href="javascript:void(0);" title="分享到 Twitter" data-sharer="twitter" data-url="https://scemsjyd.com/04_k8s_flannel/" data-title="04_K8S_核心网络插件Flannel" data-hashtags="K8S,转载"><i class="fab fa-twitter fa-fw"></i></a><a href="javascript:void(0);" title="分享到 WhatsApp" data-sharer="whatsapp" data-url="https://scemsjyd.com/04_k8s_flannel/" data-title="04_K8S_核心网络插件Flannel" data-web><i class="fab fa-whatsapp fa-fw"></i></a><a href="javascript:void(0);" title="分享到 微博" data-sharer="weibo" data-url="https://scemsjyd.com/04_k8s_flannel/" data-title="04_K8S_核心网络插件Flannel" data-image="https://upyun.cdn.scemsjyd.com/PicGo/2022/03/20220310184017.png"><i class="fab fa-weibo fa-fw"></i></a><a href="javascript:void(0);" title="分享到 Evernote" data-sharer="evernote" data-url="https://scemsjyd.com/04_k8s_flannel/" data-title="04_K8S_核心网络插件Flannel"><i class="fab fa-evernote fa-fw"></i></a></span>
            </div>
        </div>
    </div>

    <div class="post-info-more">
        <section class="post-tags"><i class="fas fa-tags fa-fw"></i>&nbsp;<a href="/tags/k8s/">K8S</a>,&nbsp;<a href="/tags/%E8%BD%AC%E8%BD%BD/">转载</a></section>
        <section>
            <span><a href="javascript:void(0);" onclick="window.history.back();">返回</a></span>&nbsp;|&nbsp;<span><a href="/">主页</a></span>
        </section>
    </div>

    <div class="post-nav"><a href="/05_k8s_%E6%A0%B8%E5%BF%83%E6%8F%92%E4%BB%B6coredns%E6%9C%8D%E5%8A%A1/" class="prev" rel="prev" title="05_K8S_核心插件coredns服务"><i class="fas fa-angle-left fa-fw"></i>05_K8S_核心插件coredns服务</a>
            <a href="/03_k8s_%E7%AE%A1%E7%90%86k8s%E6%A0%B8%E5%BF%83%E8%B5%84%E6%BA%90%E7%9A%84%E4%B8%89%E7%A7%8D%E5%9F%BA%E6%9C%AC%E6%96%B9%E6%B3%95/" class="next" rel="next" title="03_K8S_管理k8s核心资源的三种基本方法">03_K8S_管理k8s核心资源的三种基本方法<i class="fas fa-angle-right fa-fw"></i></a></div>
</div>
<div id="comments"><div id="valine" class="comment"></div><noscript>
                Please enable JavaScript to view the comments powered by <a href="https://valine.js.org/">Valine</a>.
            </noscript></div></article></div>
            </main><footer class="footer">
        <div class="footer-container"><div class="footer-line"><span><a href='https://www.upyun.com/?utm_source=lianmeng&utm_medium=referral'>本网站由<img style='vertical-align:bottom' width='60' src='/images/upyun.svg'>提供CDN加速/云存储服务</a></span></div><div class="footer-line"><i class="far fa-copyright fa-fw"></i><span itemprop="copyrightYear">2016 - 2022</span><span class="author" itemprop="copyrightHolder">&nbsp;<a href="/" target="_blank">Adam.Jin</a></span>&nbsp;|&nbsp;<span class="license"><a rel="license external nofollow noopener noreffer" href="https://creativecommons.org/licenses/by-nc/4.0/" target="_blank">CC BY-NC 4.0</a></span><span class="icp-splitter">&nbsp;|&nbsp;</span><br class="icp-br"/>
                    <span class="icp"><a href='https://beian.miit.gov.cn/'>蜀ICP备2021027971号-1</a></span></div>
        </div>
    </footer></div>

        <div id="fixed-buttons"><a href="#" id="back-to-top" class="fixed-button" title="回到顶部">
                <i class="fas fa-arrow-up fa-fw"></i>
            </a><a href="#" id="view-comments" class="fixed-button" title="查看评论">
                <i class="fas fa-comment fa-fw"></i>
            </a>
        </div><link rel="stylesheet" href="/lib/valine/valine.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/lightgallery.js@1.2.0/dist/css/lightgallery.min.css"><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/valine@1.4.14/dist/Valine.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/smooth-scroll@16.1.3/dist/smooth-scroll.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/autocomplete.js@0.37.1/dist/autocomplete.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/lunr@2.3.8/lunr.min.js"></script><script type="text/javascript" src="/lib/lunr/lunr.stemmer.support.min.js"></script><script type="text/javascript" src="/lib/lunr/lunr.zh.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/lazysizes@5.2.2/lazysizes.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/lightgallery.js@1.2.0/dist/js/lightgallery.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/lg-thumbnail.js@1.2.0/dist/lg-thumbnail.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/lg-zoom.js@1.2.0/dist/lg-zoom.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/clipboard@2.0.6/dist/clipboard.min.js"></script><script type="text/javascript" src="https://cdn.jsdelivr.net/npm/sharer.js@0.4.0/sharer.min.js"></script><script type="text/javascript">window.config={"code":{"copyTitle":"复制到剪贴板","maxShownLines":10},"comment":{"valine":{"appId":"NEWqsYTB88Sn8Pq0AsYDaXcn-gzGzoHsz","appKey":"bvPBG3pirvLqtjU6IYVxGIyP","avatar":"mp","el":"#valine","emojiCDN":"https://cdn.jsdelivr.net/npm/emoji-datasource-google@5.0.1/img/google/64/","emojiMaps":{"100":"1f4af.png","alien":"1f47d.png","anger":"1f4a2.png","angry":"1f620.png","anguished":"1f627.png","astonished":"1f632.png","black_heart":"1f5a4.png","blue_heart":"1f499.png","blush":"1f60a.png","bomb":"1f4a3.png","boom":"1f4a5.png","broken_heart":"1f494.png","brown_heart":"1f90e.png","clown_face":"1f921.png","cold_face":"1f976.png","cold_sweat":"1f630.png","confounded":"1f616.png","confused":"1f615.png","cry":"1f622.png","crying_cat_face":"1f63f.png","cupid":"1f498.png","dash":"1f4a8.png","disappointed":"1f61e.png","disappointed_relieved":"1f625.png","dizzy":"1f4ab.png","dizzy_face":"1f635.png","drooling_face":"1f924.png","exploding_head":"1f92f.png","expressionless":"1f611.png","face_vomiting":"1f92e.png","face_with_cowboy_hat":"1f920.png","face_with_hand_over_mouth":"1f92d.png","face_with_head_bandage":"1f915.png","face_with_monocle":"1f9d0.png","face_with_raised_eyebrow":"1f928.png","face_with_rolling_eyes":"1f644.png","face_with_symbols_on_mouth":"1f92c.png","face_with_thermometer":"1f912.png","fearful":"1f628.png","flushed":"1f633.png","frowning":"1f626.png","ghost":"1f47b.png","gift_heart":"1f49d.png","green_heart":"1f49a.png","grimacing":"1f62c.png","grin":"1f601.png","grinning":"1f600.png","hankey":"1f4a9.png","hear_no_evil":"1f649.png","heart":"2764-fe0f.png","heart_decoration":"1f49f.png","heart_eyes":"1f60d.png","heart_eyes_cat":"1f63b.png","heartbeat":"1f493.png","heartpulse":"1f497.png","heavy_heart_exclamation_mark_ornament":"2763-fe0f.png","hole":"1f573-fe0f.png","hot_face":"1f975.png","hugging_face":"1f917.png","hushed":"1f62f.png","imp":"1f47f.png","innocent":"1f607.png","japanese_goblin":"1f47a.png","japanese_ogre":"1f479.png","joy":"1f602.png","joy_cat":"1f639.png","kiss":"1f48b.png","kissing":"1f617.png","kissing_cat":"1f63d.png","kissing_closed_eyes":"1f61a.png","kissing_heart":"1f618.png","kissing_smiling_eyes":"1f619.png","laughing":"1f606.png","left_speech_bubble":"1f5e8-fe0f.png","love_letter":"1f48c.png","lying_face":"1f925.png","mask":"1f637.png","money_mouth_face":"1f911.png","nauseated_face":"1f922.png","nerd_face":"1f913.png","neutral_face":"1f610.png","no_mouth":"1f636.png","open_mouth":"1f62e.png","orange_heart":"1f9e1.png","partying_face":"1f973.png","pensive":"1f614.png","persevere":"1f623.png","pleading_face":"1f97a.png","pouting_cat":"1f63e.png","purple_heart":"1f49c.png","rage":"1f621.png","relaxed":"263a-fe0f.png","relieved":"1f60c.png","revolving_hearts":"1f49e.png","right_anger_bubble":"1f5ef-fe0f.png","robot_face":"1f916.png","rolling_on_the_floor_laughing":"1f923.png","scream":"1f631.png","scream_cat":"1f640.png","see_no_evil":"1f648.png","shushing_face":"1f92b.png","skull":"1f480.png","skull_and_crossbones":"2620-fe0f.png","sleeping":"1f634.png","sleepy":"1f62a.png","slightly_frowning_face":"1f641.png","slightly_smiling_face":"1f642.png","smile":"1f604.png","smile_cat":"1f638.png","smiley":"1f603.png","smiley_cat":"1f63a.png","smiling_face_with_3_hearts":"1f970.png","smiling_imp":"1f608.png","smirk":"1f60f.png","smirk_cat":"1f63c.png","sneezing_face":"1f927.png","sob":"1f62d.png","space_invader":"1f47e.png","sparkling_heart":"1f496.png","speak_no_evil":"1f64a.png","speech_balloon":"1f4ac.png","star-struck":"1f929.png","stuck_out_tongue":"1f61b.png","stuck_out_tongue_closed_eyes":"1f61d.png","stuck_out_tongue_winking_eye":"1f61c.png","sunglasses":"1f60e.png","sweat":"1f613.png","sweat_drops":"1f4a6.png","sweat_smile":"1f605.png","thinking_face":"1f914.png","thought_balloon":"1f4ad.png","tired_face":"1f62b.png","triumph":"1f624.png","two_hearts":"1f495.png","unamused":"1f612.png","upside_down_face":"1f643.png","weary":"1f629.png","white_frowning_face":"2639-fe0f.png","white_heart":"1f90d.png","wink":"1f609.png","woozy_face":"1f974.png","worried":"1f61f.png","yawning_face":"1f971.png","yellow_heart":"1f49b.png","yum":"1f60b.png","zany_face":"1f92a.png","zipper_mouth_face":"1f910.png","zzz":"1f4a4.png"},"enableQQ":false,"highlight":true,"lang":"zh-cn","pageSize":10,"placeholder":"你的评论 ...","recordIP":true,"serverURLs":"https://newqsytb.lc-cn-n1-shared.com","visitor":true}},"lightGallery":{"actualSize":false,"exThumbImage":"data-thumbnail","hideBarsDelay":2000,"selector":".lightgallery","speed":400,"thumbContHeight":80,"thumbWidth":80,"thumbnail":true},"search":{"highlightTag":"em","lunrIndexURL":"/index.json","lunrLanguageCode":"zh","lunrSegmentitURL":"/lib/lunr/lunr.segmentit.js","maxResultLength":10,"noResultsFound":"没有找到结果","snippetLength":50,"type":"lunr"}};</script><script type="text/javascript" src="/js/theme.min.js"></script></body>
</html>
